{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import copy\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(687, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No Finding</th>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <th>Lung Lesion</th>\n",
       "      <th>Lung Opacity</th>\n",
       "      <th>Edema</th>\n",
       "      <th>Consolidation</th>\n",
       "      <th>Pneumonia</th>\n",
       "      <th>Atelectasis</th>\n",
       "      <th>Pneumothorax</th>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <th>Pleural Other</th>\n",
       "      <th>Fracture</th>\n",
       "      <th>Support Devices</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>s0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>s1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>s1000</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>s101</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>s1017</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       No Finding  Enlarged Cardiomediastinum  Cardiomegaly  Lung Lesion  \\\n",
       "id                                                                         \n",
       "s0            NaN                         NaN           NaN          NaN   \n",
       "s1            NaN                         1.0           1.0          NaN   \n",
       "s1000         NaN                         NaN           NaN          NaN   \n",
       "s101          NaN                         NaN           1.0          NaN   \n",
       "s1017         NaN                         1.0           NaN          NaN   \n",
       "\n",
       "       Lung Opacity  Edema  Consolidation  Pneumonia  Atelectasis  \\\n",
       "id                                                                  \n",
       "s0              NaN    NaN            NaN        1.0          NaN   \n",
       "s1              NaN    0.0            NaN        0.0          1.0   \n",
       "s1000           1.0    NaN            NaN        NaN          NaN   \n",
       "s101            NaN    NaN            NaN        NaN          NaN   \n",
       "s1017           1.0    NaN            NaN       -1.0         -1.0   \n",
       "\n",
       "       Pneumothorax  Pleural Effusion  Pleural Other  Fracture  \\\n",
       "id                                                               \n",
       "s0              NaN               NaN            NaN       NaN   \n",
       "s1              NaN               NaN            NaN       NaN   \n",
       "s1000          -1.0               NaN            NaN       NaN   \n",
       "s101            NaN               NaN            NaN       NaN   \n",
       "s1017           NaN               NaN            NaN       NaN   \n",
       "\n",
       "       Support Devices  \n",
       "id                      \n",
       "s0                 NaN  \n",
       "s1                 NaN  \n",
       "s1000              1.0  \n",
       "s101               NaN  \n",
       "s1017              NaN  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chexpert_categories = [\"No Finding\", \"Enlarged Cardiomediastinum\", \"Cardiomegaly\",\n",
    "                      \"Lung Lesion\", \"Lung Opacity\", \"Edema\", \"Consolidation\",\n",
    "                      \"Pneumonia\", \"Atelectasis\", \"Pneumothorax\", \"Pleural Effusion\",\n",
    "                      \"Pleural Other\", \"Fracture\", \"Support Devices\"]\n",
    "\n",
    "# reports\n",
    "val = pd.read_csv('mimic_cxr_validation_reports.csv', header=None)\n",
    "val.columns = ['id', 'text']\n",
    "\n",
    "# negbio\n",
    "df_nih = pd.read_csv('mimic_cxr_validation_negbio_labeled.csv')\n",
    "df_nih.set_index('id', inplace=True)\n",
    "df_nih = df_nih[chexpert_categories]\n",
    "\n",
    "# chexpert\n",
    "df_chexpert = pd.read_csv('mimic_cxr_validation_chexpert_labeled.csv')\n",
    "df_chexpert = df_chexpert.merge(\n",
    "    val, how='inner', left_on='Reports', right_on='text'\n",
    ")\n",
    "df_chexpert.drop_duplicates(inplace=True)\n",
    "df_chexpert.set_index('id', inplace=True)\n",
    "df_chexpert.rename(columns={'Airspace Opacity': 'Lung Opacity'}, inplace=True)\n",
    "df_chexpert = df_chexpert[chexpert_categories]\n",
    "\n",
    "# ground truth\n",
    "gs = pd.read_csv('groundtruth.csv', header=0, index_col=0)\n",
    "gs.index.name = 'id'\n",
    "gs.rename(columns={'Airspace Opacity': 'Lung Opacity'}, inplace=True)\n",
    "gs = gs[chexpert_categories]\n",
    "\n",
    "print(gs.shape)\n",
    "\n",
    "# ensure all dataframes are aligned\n",
    "gs.sort_index(inplace=True)\n",
    "df_chexpert = df_chexpert.loc[gs.index]\n",
    "df_nih = df_nih.loc[gs.index]\n",
    "gs.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation function\n",
    "\n",
    "Define a helper function to evaluate the outputs in three categories: (1) mentions, (2) uncertainty, and (3) negation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_label(tar, pred, ignore_nan=False):\n",
    "    \"\"\"\n",
    "    Return precision, recall, f1, and prevalence for a single label.\n",
    "    \"\"\"\n",
    "    \n",
    "    if ignore_nan:\n",
    "        idx = ~(np.isnan(tar) | np.isnan(pred))\n",
    "        pred = pred[idx]\n",
    "        tar = tar[idx]\n",
    "    \n",
    "    results = {\n",
    "        'precision': np.nan,\n",
    "        'recall': np.nan,\n",
    "        'f1': np.nan,\n",
    "        'positives': int(tar.sum())\n",
    "    }\n",
    "    \n",
    "    if results['positives'] == 0:\n",
    "        # return NaN if no positive labels\n",
    "        return results\n",
    "    \n",
    "    results['precision'] = metrics.precision_score(tar, pred)\n",
    "    results['recall'] = metrics.recall_score(tar, pred)\n",
    "    results['f1'] = 2*(results['precision']*results['recall'])/(results['precision']+results['recall'])\n",
    "    \n",
    "    return results\n",
    "    \n",
    "\n",
    "def get_scores(target, prediction, categories, ignore_nan=False):\n",
    "    \n",
    "    \n",
    "    results = {}\n",
    "    for i, c in enumerate(categories):\n",
    "        results[c] = evaluate_label(target[:, i], prediction[:, i])\n",
    "    \n",
    "    # convert to dataframe\n",
    "    df = pd.DataFrame.from_dict(results, orient='index')\n",
    "    \n",
    "    return df\n",
    "\n",
    "def evaluate_labels(df_truth, df_label, method='mention'):\n",
    "    categories = list(df_truth.columns)\n",
    "    \n",
    "    # create the matrix of 0s and 1s\n",
    "    preds = copy.copy(df_label.values)\n",
    "    targets = copy.copy(df_truth.values)\n",
    "    \n",
    "    if method == 'mention':\n",
    "        # any mention is a 1\n",
    "        preds[np.isin(preds, [-1, 0, 1])] = 1\n",
    "        targets[np.isin(targets, [-1, 0, 1])] = 1\n",
    "\n",
    "        # no mention is a 0\n",
    "        preds[np.isnan(preds)] = 0\n",
    "        targets[np.isnan(targets)] = 0\n",
    "        \n",
    "        # do not ignore NaN (which we have set to 0 anyway)\n",
    "        ignore_nan=False\n",
    "    elif method == 'negation':\n",
    "        # successful prediction of negation\n",
    "        idxNonZero = preds != 0\n",
    "        idxZero = preds == 0\n",
    "        preds[idxNonZero] = 0\n",
    "        preds[idxZero] = 1\n",
    "        \n",
    "        idxNonZero = targets != 0\n",
    "        idxZero = targets == 0\n",
    "        targets[idxNonZero] = 0\n",
    "        targets[idxZero] = 1\n",
    "        \n",
    "        # ignore NaN values\n",
    "        ignore_nan=True\n",
    "    elif method == 'uncertain':\n",
    "        # any non-uncertain prediction is 0\n",
    "        preds[preds!= -1] = 0\n",
    "        targets[targets != -1] = 0\n",
    "        \n",
    "        # any uncertain prediction is 1\n",
    "        preds[preds == -1] = 1\n",
    "        targets[targets == -1] = 1\n",
    "        \n",
    "        # ignore NaN\n",
    "        ignore_nan=True\n",
    "    else:\n",
    "        raise ValueError(f'Unrecognized method {method}')\n",
    "        \n",
    "    df = get_scores(targets, preds, categories, ignore_nan=ignore_nan)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mention\n",
    "\n",
    "You'll note that the mention scores are approximately identical.\n",
    "\n",
    "* NegBio uses the CheXpert patterns for mention detection\n",
    "* NegBio does not use the same post-processing filter for `No Finding` that CheXpert does"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NegBio No Finding:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "precision     0.382353\n",
       "recall        0.866667\n",
       "f1            0.530612\n",
       "positives    30.000000\n",
       "Name: No Finding, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CheXpert mention:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>positives</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>No Finding</th>\n",
       "      <td>0.403</td>\n",
       "      <td>0.833</td>\n",
       "      <td>0.543</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <td>0.375</td>\n",
       "      <td>0.600</td>\n",
       "      <td>0.462</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <td>0.814</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.859</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Lesion</th>\n",
       "      <td>0.862</td>\n",
       "      <td>0.848</td>\n",
       "      <td>0.855</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Opacity</th>\n",
       "      <td>0.715</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.800</td>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edema</th>\n",
       "      <td>0.799</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.888</td>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consolidation</th>\n",
       "      <td>0.886</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.930</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumonia</th>\n",
       "      <td>0.928</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.957</td>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Atelectasis</th>\n",
       "      <td>0.893</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.944</td>\n",
       "      <td>218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumothorax</th>\n",
       "      <td>0.945</td>\n",
       "      <td>0.996</td>\n",
       "      <td>0.970</td>\n",
       "      <td>226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <td>0.968</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.973</td>\n",
       "      <td>370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Other</th>\n",
       "      <td>0.477</td>\n",
       "      <td>0.778</td>\n",
       "      <td>0.592</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fracture</th>\n",
       "      <td>0.870</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.904</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Support Devices</th>\n",
       "      <td>0.805</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.880</td>\n",
       "      <td>234</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            precision  recall     f1  positives\n",
       "No Finding                      0.403   0.833  0.543         30\n",
       "Enlarged Cardiomediastinum      0.375   0.600  0.462         70\n",
       "Cardiomegaly                    0.814   0.911  0.859        235\n",
       "Lung Lesion                     0.862   0.848  0.855         66\n",
       "Lung Opacity                    0.715   0.907  0.800        194\n",
       "Edema                           0.799   1.000  0.888        227\n",
       "Consolidation                   0.886   0.979  0.930         95\n",
       "Pneumonia                       0.928   0.987  0.957        223\n",
       "Atelectasis                     0.893   1.000  0.944        218\n",
       "Pneumothorax                    0.945   0.996  0.970        226\n",
       "Pleural Effusion                0.968   0.978  0.973        370\n",
       "Pleural Other                   0.477   0.778  0.592         27\n",
       "Fracture                        0.870   0.940  0.904         50\n",
       "Support Devices                 0.805   0.970  0.880        234"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = evaluate_labels(gs, df_nih, method='mention')\n",
    "print('NegBio No Finding:')\n",
    "display(df.loc['No Finding'])\n",
    "\n",
    "print('CheXpert mention:')\n",
    "df = evaluate_labels(gs, df_chexpert, method='mention')\n",
    "\n",
    "for c in df.columns:\n",
    "    if 'float' in str(df.dtypes[c]):\n",
    "        df[c] = np.round(df[c], 3)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Uncertain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alistairewj/miniconda3/envs/mimic-cxr/lib/python3.6/site-packages/ipykernel_launcher.py:24: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/home/alistairewj/miniconda3/envs/mimic-cxr/lib/python3.6/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">precision</th>\n",
       "      <th colspan=\"2\" halign=\"left\">recall</th>\n",
       "      <th colspan=\"2\" halign=\"left\">f1</th>\n",
       "      <th>positives</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>NegBio</th>\n",
       "      <th>CheXpert</th>\n",
       "      <th>NegBio</th>\n",
       "      <th>CheXpert</th>\n",
       "      <th>NegBio</th>\n",
       "      <th>CheXpert</th>\n",
       "      <th>NegBio</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Uncertainty</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>No Finding</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <td>0.033</td>\n",
       "      <td>0.036</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.065</td>\n",
       "      <td>0.069</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <td>0.156</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Lesion</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Opacity</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edema</th>\n",
       "      <td>0.102</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.600</td>\n",
       "      <td>0.169</td>\n",
       "      <td>0.207</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consolidation</th>\n",
       "      <td>0.529</td>\n",
       "      <td>0.273</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.176</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.214</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumonia</th>\n",
       "      <td>0.432</td>\n",
       "      <td>0.407</td>\n",
       "      <td>0.613</td>\n",
       "      <td>0.565</td>\n",
       "      <td>0.507</td>\n",
       "      <td>0.473</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Atelectasis</th>\n",
       "      <td>0.333</td>\n",
       "      <td>0.289</td>\n",
       "      <td>0.706</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.453</td>\n",
       "      <td>0.400</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumothorax</th>\n",
       "      <td>0.375</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.167</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <td>0.432</td>\n",
       "      <td>0.414</td>\n",
       "      <td>0.889</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.582</td>\n",
       "      <td>0.511</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Other</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fracture</th>\n",
       "      <td>0.500</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Support Devices</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           precision          recall              f1           \\\n",
       "                              NegBio CheXpert NegBio CheXpert NegBio CheXpert   \n",
       "Uncertainty                                                                     \n",
       "No Finding                       NaN      NaN    NaN      NaN    NaN      NaN   \n",
       "Enlarged Cardiomediastinum     0.033    0.036  1.000    1.000  0.065    0.069   \n",
       "Cardiomegaly                   0.156    0.000  0.500    0.000  0.237      NaN   \n",
       "Lung Lesion                    0.000    0.000  0.000    0.000    NaN      NaN   \n",
       "Lung Opacity                     NaN      NaN    NaN      NaN    NaN      NaN   \n",
       "Edema                          0.102    0.125  0.500    0.600  0.169    0.207   \n",
       "Consolidation                  0.529    0.273  0.529    0.176  0.529    0.214   \n",
       "Pneumonia                      0.432    0.407  0.613    0.565  0.507    0.473   \n",
       "Atelectasis                    0.333    0.289  0.706    0.647  0.453    0.400   \n",
       "Pneumothorax                   0.375    0.250  0.375    0.125  0.375    0.167   \n",
       "Pleural Effusion               0.432    0.414  0.889    0.667  0.582    0.511   \n",
       "Pleural Other                    NaN      NaN    NaN      NaN    NaN      NaN   \n",
       "Fracture                       0.500    0.000  0.500    0.000  0.500      NaN   \n",
       "Support Devices                  NaN      NaN    NaN      NaN    NaN      NaN   \n",
       "\n",
       "                           positives  \n",
       "                              NegBio  \n",
       "Uncertainty                           \n",
       "No Finding                         0  \n",
       "Enlarged Cardiomediastinum         1  \n",
       "Cardiomegaly                      14  \n",
       "Lung Lesion                        8  \n",
       "Lung Opacity                       0  \n",
       "Edema                             10  \n",
       "Consolidation                     17  \n",
       "Pneumonia                         62  \n",
       "Atelectasis                       17  \n",
       "Pneumothorax                       8  \n",
       "Pleural Effusion                  18  \n",
       "Pleural Other                      0  \n",
       "Fracture                           2  \n",
       "Support Devices                    0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = evaluate_labels(gs, df_nih, method='uncertain')\n",
    "df.columns = pd.MultiIndex.from_tuples([('NegBio', c) for c in df.columns])\n",
    "\n",
    "cx = evaluate_labels(gs, df_chexpert, method='uncertain')\n",
    "cx.columns = pd.MultiIndex.from_tuples([('CheXpert', c) for c in cx.columns])\n",
    "\n",
    "df = df.merge(cx, how='inner', left_index=True, right_index=True)\n",
    "# df.columns.swaplabel(0, 1, axis=1, inplace=True)\n",
    "df.columns = df.columns.reorder_levels([1, 0])\n",
    "\n",
    "# re-order columns\n",
    "df = df[['precision', 'recall', 'f1', 'positives']]\n",
    "\n",
    "# round values\n",
    "for c in df.columns:\n",
    "    if 'float' in str(df.dtypes[c]):\n",
    "        df[c] = np.round(df[c], 3)\n",
    "\n",
    "# drop the unecessary final column\n",
    "df.drop(('positives', 'CheXpert'), axis=1, inplace=True)\n",
    "\n",
    "# output to latex\n",
    "df.index.name = 'Uncertainty'\n",
    "\n",
    "df.to_latex('uncertainty.tex')\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Negation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alistairewj/miniconda3/envs/mimic-cxr/lib/python3.6/site-packages/ipykernel_launcher.py:24: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">precision</th>\n",
       "      <th colspan=\"2\" halign=\"left\">recall</th>\n",
       "      <th colspan=\"2\" halign=\"left\">f1</th>\n",
       "      <th>positives</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>NegBio</th>\n",
       "      <th>CheXpert</th>\n",
       "      <th>NegBio</th>\n",
       "      <th>CheXpert</th>\n",
       "      <th>NegBio</th>\n",
       "      <th>CheXpert</th>\n",
       "      <th>NegBio</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Negation</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>No Finding</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <td>0.654</td>\n",
       "      <td>0.654</td>\n",
       "      <td>0.607</td>\n",
       "      <td>0.607</td>\n",
       "      <td>0.630</td>\n",
       "      <td>0.630</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <td>0.855</td>\n",
       "      <td>0.855</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.781</td>\n",
       "      <td>0.781</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Lesion</th>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Opacity</th>\n",
       "      <td>0.429</td>\n",
       "      <td>0.533</td>\n",
       "      <td>0.391</td>\n",
       "      <td>0.348</td>\n",
       "      <td>0.409</td>\n",
       "      <td>0.421</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edema</th>\n",
       "      <td>0.713</td>\n",
       "      <td>0.714</td>\n",
       "      <td>0.847</td>\n",
       "      <td>0.824</td>\n",
       "      <td>0.774</td>\n",
       "      <td>0.765</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consolidation</th>\n",
       "      <td>0.917</td>\n",
       "      <td>0.917</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.936</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumonia</th>\n",
       "      <td>0.836</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.735</td>\n",
       "      <td>0.711</td>\n",
       "      <td>0.782</td>\n",
       "      <td>0.781</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Atelectasis</th>\n",
       "      <td>0.333</td>\n",
       "      <td>0.300</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.462</td>\n",
       "      <td>0.429</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumothorax</th>\n",
       "      <td>0.919</td>\n",
       "      <td>0.926</td>\n",
       "      <td>0.955</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.918</td>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <td>0.906</td>\n",
       "      <td>0.919</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.940</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Other</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fracture</th>\n",
       "      <td>0.600</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.462</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Support Devices</th>\n",
       "      <td>0.200</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.267</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           precision          recall              f1           \\\n",
       "                              NegBio CheXpert NegBio CheXpert NegBio CheXpert   \n",
       "Negation                                                                        \n",
       "No Finding                       NaN      NaN    NaN      NaN    NaN      NaN   \n",
       "Enlarged Cardiomediastinum     0.654    0.654  0.607    0.607  0.630    0.630   \n",
       "Cardiomegaly                   0.855    0.855  0.720    0.720  0.781    0.781   \n",
       "Lung Lesion                    0.500    0.500  0.500    0.500  0.500    0.500   \n",
       "Lung Opacity                   0.429    0.533  0.391    0.348  0.409    0.421   \n",
       "Edema                          0.713    0.714  0.847    0.824  0.774    0.765   \n",
       "Consolidation                  0.917    0.917  0.957    0.957  0.936    0.936   \n",
       "Pneumonia                      0.836    0.868  0.735    0.711  0.782    0.781   \n",
       "Atelectasis                    0.333    0.300  0.750    0.750  0.462    0.429   \n",
       "Pneumothorax                   0.919    0.926  0.955    0.911  0.937    0.918   \n",
       "Pleural Effusion               0.906    0.919  0.939    0.963  0.922    0.940   \n",
       "Pleural Other                  0.000    0.000  0.000    0.000    NaN      NaN   \n",
       "Fracture                       0.600    0.000  0.375    0.000  0.462      NaN   \n",
       "Support Devices                0.200    0.000  0.400    0.000  0.267      NaN   \n",
       "\n",
       "                           positives  \n",
       "                              NegBio  \n",
       "Negation                              \n",
       "No Finding                         0  \n",
       "Enlarged Cardiomediastinum        28  \n",
       "Cardiomegaly                      82  \n",
       "Lung Lesion                        4  \n",
       "Lung Opacity                      23  \n",
       "Edema                             85  \n",
       "Consolidation                     23  \n",
       "Pneumonia                         83  \n",
       "Atelectasis                        4  \n",
       "Pneumothorax                     179  \n",
       "Pleural Effusion                  82  \n",
       "Pleural Other                      2  \n",
       "Fracture                           8  \n",
       "Support Devices                    5  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = evaluate_labels(gs, df_nih, method='negation')\n",
    "df.columns = pd.MultiIndex.from_tuples([('NegBio', c) for c in df.columns])\n",
    "\n",
    "cx = evaluate_labels(gs, df_chexpert, method='negation')\n",
    "cx.columns = pd.MultiIndex.from_tuples([('CheXpert', c) for c in cx.columns])\n",
    "\n",
    "df = df.merge(cx, how='inner', left_index=True, right_index=True)\n",
    "# df.columns.swaplabel(0, 1, axis=1, inplace=True)\n",
    "df.columns = df.columns.reorder_levels([1, 0])\n",
    "\n",
    "# re-order columns\n",
    "df = df[['precision', 'recall', 'f1', 'positives']]\n",
    "\n",
    "# round values\n",
    "for c in df.columns:\n",
    "    if 'float' in str(df.dtypes[c]):\n",
    "        df[c] = np.round(df[c], 3)\n",
    "\n",
    "# drop the unecessary final column (redundant)\n",
    "df.drop(('positives', 'CheXpert'), axis=1, inplace=True)\n",
    "\n",
    "# output to latex\n",
    "df.index.name = 'Negation'\n",
    "\n",
    "df.to_latex('negation.tex')\n",
    "\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mimic-cxr",
   "language": "python",
   "name": "mimic-cxr"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
